{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 利用神经网络对MNIST进行优化"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1 导入MNIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"A very simple MNIST classifier.\n",
    "See extensive documentation at\n",
    "https://www.tensorflow.org/get_started/mnist/beginners\n",
    "\"\"\"\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "FLAGS = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-2-698ada706af1>:3: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From C:\\Users\\hplxg\\Anaconda3\\envs\\python3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From C:\\Users\\hplxg\\Anaconda3\\envs\\python3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\Users\\hplxg\\Anaconda3\\envs\\python3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\Users\\hplxg\\Anaconda3\\envs\\python3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\Users\\hplxg\\Anaconda3\\envs\\python3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "# Import data\n",
    "data_dir = '/tmp/tensorflow/mnist/input_data'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2 构建模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "# W1 = tf.Variable(tf.zeros([784, 10]))\n",
    "# b1 = tf.Variable(tf.zeros([10]))\n",
    "# y = tf.matmul(x, W) + b"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1 增加隐层，以提高准确率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义一个增加隐层的方法\n",
    "\n",
    "def add_layer(inputs, in_size, out_size, activation_function=None):\n",
    "    loc_w = tf.Variable(tf.random_normal([in_size, out_size], stddev=0.1))       \n",
    "    loc_b = tf.Variable(tf.zeros([out_size]) + 0.1)  \n",
    "    loc_y = tf.matmul(inputs, loc_w) + loc_b\n",
    "    if activation_function is None:\n",
    "        loc_outputs = loc_y\n",
    "    else:\n",
    "        loc_outputs = activation_function(loc_y)\n",
    "    return loc_outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 增加隐层\n",
    "l_1 = add_layer(x, 784,700, activation_function=tf.nn.sigmoid)  # 隐藏层,使用激励函数sigmoid\n",
    "l_2 = add_layer(l_1, 700,600, activation_function=None)  \n",
    "#l_3 = add_layer(l_2, 600,500, activation_function=None)  # 增加两层隐层, 并适当增加每层的神经元数量\n",
    "y = add_layer(l_2, 600, 10, activation_function=None)  # 输出层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define loss and optimizer\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-7-cf2c854ecc80>:2: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See `tf.nn.softmax_cross_entropy_with_logits_v2`.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 计算交叉熵\n",
    "cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "\n",
    "# cross_entropy = tf.losses.sigmoid_cross_entropy(multi_class_labels=y_, logits=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 生成一个step\n",
    "lr = tf.Variable(0.001, dtype=tf.float32) \n",
    "\n",
    "train_step = tf.train.AdamOptimizer(lr).minimize(cross_entropy)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter0, Testing Accuracy0.9206, Learning_rate0.001\n",
      "Iter1, Testing Accuracy0.9357, Learning_rate0.00095\n",
      "Iter2, Testing Accuracy0.9575, Learning_rate0.0009025\n",
      "Iter3, Testing Accuracy0.9618, Learning_rate0.000857375\n",
      "Iter4, Testing Accuracy0.9669, Learning_rate0.00081450626\n",
      "Iter5, Testing Accuracy0.9673, Learning_rate0.0007737809\n",
      "Iter6, Testing Accuracy0.9716, Learning_rate0.0007350919\n",
      "Iter7, Testing Accuracy0.9698, Learning_rate0.0006983373\n",
      "Iter8, Testing Accuracy0.9742, Learning_rate0.0006634204\n",
      "Iter9, Testing Accuracy0.9696, Learning_rate0.0006302494\n",
      "Iter10, Testing Accuracy0.9798, Learning_rate0.0005987369\n",
      "Iter11, Testing Accuracy0.9753, Learning_rate0.0005688001\n",
      "Iter12, Testing Accuracy0.9769, Learning_rate0.0005403601\n",
      "Iter13, Testing Accuracy0.9796, Learning_rate0.0005133421\n",
      "Iter14, Testing Accuracy0.979, Learning_rate0.000487675\n",
      "Iter15, Testing Accuracy0.98, Learning_rate0.00046329122\n",
      "Iter16, Testing Accuracy0.9799, Learning_rate0.00044012666\n",
      "Iter17, Testing Accuracy0.9762, Learning_rate0.00041812033\n",
      "Iter18, Testing Accuracy0.9817, Learning_rate0.00039721432\n",
      "Iter19, Testing Accuracy0.9816, Learning_rate0.0003773536\n",
      "Iter20, Testing Accuracy0.9796, Learning_rate0.00035848594\n",
      "Iter21, Testing Accuracy0.9798, Learning_rate0.00034056162\n",
      "Iter22, Testing Accuracy0.9794, Learning_rate0.00032353355\n",
      "Iter23, Testing Accuracy0.9783, Learning_rate0.00030735688\n",
      "Iter24, Testing Accuracy0.9802, Learning_rate0.000291989\n",
      "Iter25, Testing Accuracy0.9803, Learning_rate0.00027738957\n",
      "Iter26, Testing Accuracy0.981, Learning_rate0.0002635201\n",
      "Iter27, Testing Accuracy0.9809, Learning_rate0.00025034408\n",
      "Iter28, Testing Accuracy0.9788, Learning_rate0.00023782688\n",
      "Iter29, Testing Accuracy0.9806, Learning_rate0.00022593554\n",
      "Iter30, Testing Accuracy0.9812, Learning_rate0.00021463877\n",
      "Iter31, Testing Accuracy0.9818, Learning_rate0.00020390682\n",
      "Iter32, Testing Accuracy0.9816, Learning_rate0.00019371149\n",
      "Iter33, Testing Accuracy0.9817, Learning_rate0.0001840259\n",
      "Iter34, Testing Accuracy0.9817, Learning_rate0.00017482461\n",
      "Iter35, Testing Accuracy0.9809, Learning_rate0.00016608338\n",
      "Iter36, Testing Accuracy0.9802, Learning_rate0.00015777921\n",
      "Iter37, Testing Accuracy0.9817, Learning_rate0.00014989026\n",
      "Iter38, Testing Accuracy0.9814, Learning_rate0.00014239574\n",
      "Iter39, Testing Accuracy0.9816, Learning_rate0.00013527596\n",
      "Iter40, Testing Accuracy0.9817, Learning_rate0.00012851215\n",
      "Iter41, Testing Accuracy0.9818, Learning_rate0.00012208655\n",
      "Iter42, Testing Accuracy0.9813, Learning_rate0.00011598222\n",
      "Iter43, Testing Accuracy0.9821, Learning_rate0.00011018311\n",
      "Iter44, Testing Accuracy0.9816, Learning_rate0.000104673956\n",
      "Iter45, Testing Accuracy0.9818, Learning_rate9.944026e-05\n",
      "Iter46, Testing Accuracy0.9815, Learning_rate9.446825e-05\n",
      "Iter47, Testing Accuracy0.9817, Learning_rate8.974483e-05\n",
      "Iter48, Testing Accuracy0.9814, Learning_rate8.525759e-05\n",
      "Iter49, Testing Accuracy0.9817, Learning_rate8.099471e-05\n"
     ]
    }
   ],
   "source": [
    "# 进行训练\n",
    "# Train\n",
    "batch_size = 100\n",
    "n_batch = mnist.train.num_examples//batch_size\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    \n",
    "    for epoch in range(50):\n",
    "        sess.run(tf.assign(lr, 0.001 * (0.95 ** epoch)))\n",
    "        for batch in range(n_batch):\n",
    "                    \n",
    "        \n",
    "                #获得一个批次\n",
    "            batch_xs, batch_ys = mnist.train.next_batch(batch_size)\n",
    "            sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "        \n",
    "        learning_rate = sess.run(lr) # 设置学习率\n",
    "        #训练完一个周期后测试数据准确率\n",
    "        correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "        accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "        acc = sess.run(accuracy, feed_dict={x: mnist.test.images,y_: mnist.test.labels})\n",
    "        \n",
    "        print(\"Iter\" + str(epoch) + \", Testing Accuracy\" + str(acc) + \", Learning_rate\" + str(learning_rate))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
